import requests
import json
import pymongo

# 连接MongoDB数据库
def connect_mongodb():
    try:
        client = pymongo.MongoClient('mongodb://localhost:27017/')
        return client
    except pymongo.errors.ConnectionFailure as e:
        print(f"连接MongoDB失败: {e}")
        return None

# 发送HTTP请求获取数据
def fetch_data(url, headers):
    try:
        response = requests.get(url, headers=headers)
        response.raise_for_status()  # 检查HTTP响应状态码
        return json.loads(response.text)
    except requests.RequestException as e:
        print(f"请求失败: {e}")
        return None
    except json.JSONDecodeError as e:
        print(f"解析JSON数据失败: {e}")
        return None

# 处理数据并插入到数据库
def process_and_insert_data(collection, html):
    data_to_insert = []
    for zq in html['data']:
        data_to_insert.append({
            "债券名称": zq['ZQ_NAME'],
            "债券编码": zq['ZQ_CODE'],
            "发行地区": zq['AD_NAME'],
            "发行时间": zq['ZQ_FXTIME'],
            "发行期限": zq['ZQQX_NAME'],
            "发行规模": zq['FX_AMT'],
            "利率": zq['LL'],
            "付息方式": zq['FXFS'],
            "起息日": zq['QX_DATE']
        })
        print(f"正在爬取债券{zq['ZQ_CODE']}......")
    if data_to_insert:
        collection.insert_many(data_to_insert)

# 主函数
def main():
    client = connect_mongodb()
    if client is None:
        return

    db_name = '中国地方政府债券'
    client.drop_database(db_name)
    db = client[db_name]
    collection = db['data']

    headers = {
        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
    }

    for page in range(1, 1642):
        url = f"https://www.governbond.org.cn:4443/api/loadBondData.action?timeStamp=1744278770234&dataType=ZQFXLISTBYAD&adList=&adCode=87&zqlx=&year=&fxfs=&qxr=&fxqx=&zqCode=&zqName=&page={page}&pageSize=10"
        html = fetch_data(url, headers)
        if html:
            process_and_insert_data(collection, html)

    client.close()

if __name__ == "__main__":
    main()